<!DOCTYPE html>
<html lang="zh-cn">

<head>
  <meta charset="utf-8">
  <meta name="author" content="wushen_zhong" />
  <meta name="viewport" content="width=device-width" />
  <title>Example</title>
  <style>
    a {
      color: blue;
      cursor: pointer;
    }

    table {
      border-collapse: collapse;

      th,
      td {
        border: solid;
      }
    }
  </style>
</head>

<body>

  <main id="root">
    <h1>爬取工具</h1>
    <p>
      可爬取：
    </p>
    <table>
      <thead>
        <tr>
          <th>类型</th>
          <th>链接格式</th>
          <th>状态</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <td>B站视频</td>
          <td>https://www.bilibili.com/video...</td>
          <td>已测试</td>
        </tr>
        <tr>
          <td>B站视频合集</td>
          <td>https://space.bilibili.com/.../lists/...</td>
          <td>已测试</td>
        </tr>
        <tr>
          <td>腾讯会议分享回放</td>
          <td>https://meeting.tencent.com/user-center/shared-record-info...</td>
          <td></td>
        </tr>
        <tr>
          <td>微信公众号视频</td>
          <td>https://mp.weixin.qq.com/s/...</td>
          <td></td>
        </tr>
        <tr>
          <td>微信公众号【考研竞赛数学】数学分析考研各文章的图片</td>
          <td>固定，请在链接输入框内输入1</td>
          <td></td>
        </tr>
        <tr>
          <td>小红书文章及评论</td>
          <td>https://www.xiaohongshu.com/search_result...</td>
          <td>已测试</td>
        </tr>
      </tbody>
    </table>
    <small>注：下载腾讯会议分享回放配合ModHeader使用更佳</small>
    <p>
      确保已安装<a onclick="window.open('https://gitee.com/wushen_zhong/browsers-crawler/raw/master/userscript.js')">所需脚本</a><br />
      <a onclick="window.open(); window.open()">测试打开两个页面</a><br />
      地址栏右侧允许打开多个页面<br />
      选择空文件夹来保存文件<br />
      运行时不要关闭自动打开的页面
    </p>
    <label>链接：</label><input id="url" type="url" required /><br />
    <label>最大标签页数：</label><input id="num" type="number" min="1" placeholder="默认为1" /><br />
    <button id="run">运行</button>
    <hr>
  </main>

  <script type="module">

    /**
     * https://developer.mozilla.org/en-US/docs/Web/API/setTimeout#reasons_for_delays_longer_than_specified
     */
    class Timer extends EventTarget {
      #delay
      #startTime
      #timeoutId
      #hasElapsedTime = 0
      static #baseId = 0
      static #timeoutIdMap = new Map()
      static #intervalIdMap = new Map()
      constructor(delay) {
        super()
        this.#delay = delay
      }
      start() {
        this.#startTime = new Date().getTime()
        const handle = () => {
          this.#hasElapsedTime += new Date().getTime() - this.#startTime
          if (this.#delay < this.#hasElapsedTime)
            this.dispatchEvent(new CustomEvent('timeout'))
          else
            this.#timeoutId = setTimeout(handle, this.#delay - this.#hasElapsedTime)
        }
        handle()
      }
      reset() {
        this.#startTime = new Date().getTime()
        this.#hasElapsedTime = 0
      }
      stop() {
        clearTimeout(this.#timeoutId)
        this.#hasElapsedTime += new Date().getTime() - this.#startTime
      }
      static setTimeout(handle, delay) {
        const timer = new Timer(delay)
        timer.addEventListener('timeout', handle)
        timer.start()
        const id = Timer.#baseId++
        Timer.#timeoutIdMap.set(id, timer)
        return id
      }
      static setInterval(handle, delay) {
        const timer = new Timer(delay)
        timer.addEventListener('timeout', () => {
          handle()
          timer.reset()
          timer.start()
        })
        timer.start()
        const id = Timer.#baseId++
        Timer.#intervalIdMap.set(id, timer)
        return id
      }
      static clearTimeout(id) {
        Timer.#timeoutIdMap.get(id).stop()
        Timer.#timeoutIdMap.delete(id)
      }
      static clearInterval(id) {
        Timer.#intervalIdMap.get(id).stop()
        Timer.#intervalIdMap.delete(id)
      }
      static wait(delay) {
        const { promise, resolve } = Promise.withResolvers()
        const timer = new Timer(delay)
        timer.addEventListener('timeout', resolve)
        timer.start()
        return promise
      }
    }

    /**
     * All the threads in pool are promise chains
     */
    class FakeThreadPool {
      #maxThreads
      #threads = new Set()
      #taskList = []
      #pauseSign = false

      /**
       * Create a FakeThreadPool
       * @constructor
       * @param {number} maxThreads 
       */
      constructor(maxThreads) {
        if (typeof maxThreads != 'number')
          throw new TypeError('"maxThreads" is not a number')
        this.#maxThreads = maxThreads
      }

      async #createTread() {
        while (this.#taskList.length !== 0 && !this.#pauseSign) {
          const [handle, resolve, reject] = this.#taskList.shift()
          try {
            resolve(await handle())
          } catch (err) {
            reject(err)
          }
        }
      }

      async #startThread() {
        const thread = this.#createTread()
        this.#threads.add(thread)
        await thread
        this.#threads.delete(thread)
      }

      /**
       * Add a task to the task list and run later
       * @param {function} handle 
       * @returns {Promise} Resolves with the result of handle
       */
      submit(handle) {
        const { promise, resolve, reject } = Promise.withResolvers()
        this.#taskList.push([handle, resolve, reject])
        if (this.#threads.size < this.#maxThreads && !this.#pauseSign)
          this.#startThread()
        return promise
      }

      /**
       * Pause all the thread
       * @returns {FakeThreadPool}  Itself
       */
      async pause() {
        this.#pauseSign = true
        await Promise.allSettled(this.#threads)
        return this
      }

      /**
       * Create threads with min value between maxThreads and the number of tasks
       * @returns {FakeThreadPool}  Itself
       */
      async resume() {
        this.#pauseSign = false
        for (let count = this.#threads.size < this.#taskList.length ? this.#threads.size : this.#taskList.length; count > 0; count--)
          this.#startThread()
        return this
      }

      shutdown(timeout = 10000) {
        let isNotCleared = false
        const handle = () => {
          if (isNotCleared) {
            isNotCleared = true
            this.#threads.clear()
            this.#taskList = []
          }
        }
        return Promise.any([
          Timer.wait(timeout).then(handle),
          this.pause().then(handle)
        ])
      }
    }

    class TabSession extends EventTarget {
      static #label = 'using crawler in browser'
      static #idCache = 0
      #tab
      #id
      #heartbeatIntervalId
      constructor() { super() }
      connect(url, timeout = 10000) {
        this.#tab = window.open(url)
        if (!this.#tab) {
          this.dispatchEvent(new CustomEvent('error', { detail: new Error('opened url failed') }))
          return
        }
        window.focus()
        if (timeout >= 0)
          var connectTimeoutId = Timer.setTimeout(() => this.dispatchEvent(new CustomEvent('timeout', { detail: new Error('timeout occurred during connection') })), timeout)
        this.#id = TabSession.#idCache++
        const heartbeatHandle = () => this.#tab.postMessage({ label: TabSession.#label, id: this.#id, heartbeat: true }, '*')
        const handle = ({ data: { label, id } }) => {
          if (label !== TabSession.#label || id !== this.#id)
            return
          if (timeout >= 0) {
            Timer.clearTimeout(connectTimeoutId)
            var timer = new Timer(timeout)
            timer.addEventListener('timeout', () => this.dispatchEvent(new CustomEvent('timeout', { detail: new Error('timeout occurred while keeping the connection') })))
            timer.start()
          }
          Timer.clearInterval(this.#heartbeatIntervalId)
          window.removeEventListener('message', handle)
          this.dispatchEvent(new CustomEvent('connect'))
          window.addEventListener('message', ({ data: { label, id, heartbeat, data: detail } }) => {
            if (label != TabSession.#label || id !== this.#id)
              return
            if (timeout >= 0)
              timer.reset()
            if (heartbeat)
              return
            this.dispatchEvent(new CustomEvent('message', { detail }))
          })
          this.#heartbeatIntervalId = Timer.setInterval(heartbeatHandle, 1000)
        }
        window.addEventListener('message', handle)
        this.#heartbeatIntervalId = Timer.setInterval(heartbeatHandle, 200)
      }
      close() { }
      destory() {
        Timer.clearInterval(this.#heartbeatIntervalId)
        if (this.#tab) {
          this.#tab.close()
          this.#tab = undefined
        }
      }
      send(data) {
        this.#tab.postMessage({ label: TabSession.#label, id: this.#id, data }, '*')
      }
    }

    function getFromNewTab(url, handles, threadPool, timeout = 10000) {
      const session = new TabSession()
      const isFunc = typeof handles == "function"
      if (isFunc)
        handles = { value: handles }
      const promises = {}, resolves = {}, rejects = {}
      for (const key in handles) {
        const { promise, resolve, reject } = Promise.withResolvers()
        promises[key] = promise
        resolves[key] = resolve
        rejects[key] = reject
      }
      session.addEventListener('connect', () => {
        for (const key in handles)
          session.send({ key: key, handle: handles[key].toString() })
      })
      const logMap = new Map()
      session.addEventListener('message', async ({ detail: { type, error, key, data } }) => {
        if (type == 'result') {
          if (error == undefined)
            resolves[key](data)
          else
            rejects[key](error)
        } else if (type == 'log') {
          logMap.set(key, data)
          console.table([...logMap.values()])
          // if (data?.progress == 100)
          //   logMap.delete(id)
        }
      })
      session.addEventListener('timeout', ({ detail }) => {
        for (const key in rejects)
          rejects[key](detail)
      })
      session.addEventListener('error', ({ detail }) => {
        for (const key in rejects)
          rejects[key](detail)
      })
      const task = async () => {
        session.connect(url, timeout)
        await Promise.allSettled(Object.values(promises))
        session.destory()
      }
      if (threadPool)
        threadPool.submit(task)
      else
        task()
      return isFunc ? promises.value : promises
    }


    function toValidFileName(name) { return name.replaceAll(/[\\\/\:\?\"\<\>\*\|\n]/g, ' ').trim() }

    function save(directoryHandle, name, blob) {
      return directoryHandle.getFileHandle(name, { create: true })
        .then(fileHandle => fileHandle.createWritable())
        .then(writableFileStream => {
          writableFileStream.write(blob)
          writableFileStream.close()
        })
    }

    async function bilibiliVideo(fileSystemDirectoryHandle, url, fakeThreadPool) {
      let { title, cover, video, audio } = getFromNewTab(url, {
        title: () => gather(() => document?.querySelector('h1.video-title')?.title),
        cover: () => gather(() => document?.querySelector("div > div.dropdown-top > div.dropdown-top-left > div.video-capture > div.video-capture-img.b-img > picture > img")?.src).then(url => get(url.replace(/@.*?$/, ''))),
        video: () => gather(() => __playinfo__?.data?.dash?.video?.[0]?.baseUrl).then(url => get(url)),
        audio: () => gather(() => __playinfo__?.data?.dash?.audio?.[0]?.baseUrl).then(url => get(url))
      }, fakeThreadPool)
      title = await title
      root.insertAdjacentHTML('beforeend', `<p>视频名称：${title}</p>`)
      const directoryHandle = await fileSystemDirectoryHandle.getDirectoryHandle(toValidFileName(title), { create: true })
      await Promise.allSettled([
        cover.then(c => save(directoryHandle, 'cover.jpg', c)),
        video.then(v => save(directoryHandle, 'video.mp4', v)),
        audio.then(a => save(directoryHandle, 'audio.mp4', a))
      ])
    }

    async function bilibiliCollection(fileSystemDirectoryHandle, url, fakeThreadPool) {
      let { title, urls } = getFromNewTab(url, {
        title: () => gather(() => document?.querySelector("#app > main > div.space-list-details > div.list-header > div.title > div.name")?.innerText),
        urls: async () => {
          let urls = []
          const nextButton = await gather(() => document?.querySelector("#app > main > div.space-list-details > div.list-footer > div > div.vui_pagenation--btns > button:nth-last-child(1)"), [null])
          console.log(nextButton)
          while (true) {
            await scrollToEnd()
            const as = [...document.querySelectorAll('a.bili-cover-card')]
            urls = urls.concat(as.map(a => a.href))
            if (nextButton.classList.contains('vui_button--disabled'))
              break
            else
              nextButton.click()
          }
          return urls
        }
      })
      title = await title
      root.insertAdjacentHTML('beforeend', `<p>名称：${title}</p>`)
      const rootDirectoryHandle = await fileSystemDirectoryHandle.getDirectoryHandle(toValidFileName(title), { create: true })
      urls = await urls
      root.insertAdjacentHTML('beforeend', `<p>视频数量：${urls.length}</p>`)
      await Promise.all(urls.map(url => bilibiliVideo(rootDirectoryHandle, url, fakeThreadPool)))
    }

    async function tencentMeetingRecord(fileSystemDirectoryHandle, url, fakeThreadPool) {
      let { title, video, subVideo } = getFromNewTab(url, {
        title: () => gather(() => document?.querySelector("#cloud-record-detail-body-content > div > div.cloud-header-container > div > div.header-left > div > div.subject-container > div > div > span")?.innerHTML),
        video: () => gather(() => document?.querySelector('#test-video_html5_api')?.src, [undefined, '']).then(get),
        subVideo: async () => {
          await wait(6000)
          const src = document?.querySelector('#sub-main-player-video')?.src
          if (src !== undefined && src != '')
            return get(src)
        }
      }, fakeThreadPool)
      title = await title
      root.insertAdjacentHTML('beforeend', `<p>视频名称：${title}</p>`)
      if (!await video) {
        video = getFromNewTab(url, () => gather(() => document?.querySelector('#test-video_html5_api')?.src, [undefined, '']).then(src => download('说话人.mp4', src)), fakeThreadPool)
        subVideo = getFromNewTab(url, async () => {
          await wait(6000)
          const src = document?.querySelector('#sub-main-player-video')?.src
          if (src !== undefined && src != '')
            return download('共享屏幕.mp4', src)
        }, fakeThreadPool)
        await video
        await subVideo
        return
      }
      const directoryHandle = await fileSystemDirectoryHandle.getDirectoryHandle(toValidFileName(title), { create: true })
      await Promise.all([
        video.then(v => save(directoryHandle, '说话人.mp4', v)),
        subVideo.then(s => save(directoryHandle, '共享屏幕.mp4', s))
      ])
    }

    async function weChatOfficialAccount(fileSystemDirectoryHandle, url, fakeThreadPool) {
      let { title, video } = getFromNewTab(url, {
        title: () => gather(() => document?.querySelector('#js_video_page_title')?.innerText),
        video: () => gather(() => document?.querySelector('video')?.src).then(src => get(src))
      }, fakeThreadPool)
      title = await title
      root.insertAdjacentHTML('beforeend', `<p>视频名称：${title}</p>`)
      save(fileSystemDirectoryHandle, title + '.mp4', await video)
    }

    async function weChatOfficialAccountImages(fileSystemDirectoryHandle, url, fakeThreadPool) {
      const html = await getFromNewTab(
        'https://mp.weixin.qq.com/mp/appmsgalbum?__biz=MzI2OTE2NzczNQ==&action=getalbum&album_id=1843712419781476353',
        () => scrollToEnd().then(() => document.documentElement.outerHTML),
        fakeThreadPool
      )
      await Promise.allSettled(
        [...html.matchAll(/(?<=data-link=")http.*?(?=")/g)]
          .map(matched => matched[0])
          .map(url => 'https' + url.slice(4))
          .map(async url => {
            let { title, imgs } = getFromNewTab(url, {
              title: () => document?.querySelector('#activity-name')?.innerText,
              imgs: () => Promise.all(
                [...document.querySelectorAll('img')]
                  .map(img => img.dataset.src)
                  .filter(src => src !== undefined)
                  .map(src => get(src))
              )
            }, fakeThreadPool)
            title = await title
            root.insertAdjacentHTML('beforeend', `<p>标题：${title}</p>`)
            const directoryHandle = await fileSystemDirectoryHandle.getDirectoryHandle(toValidFileName(title), { create: true });
            (await imgs).entries().forEach(([index, img]) => save(directoryHandle, index + '.' + img.type.slice(6), img))
          })
          .map(promise => promise.catch(err => root.insertAdjacentHTML('beforeend', `<p>发生错误：${err}</p>`)))
      )
    }

    async function bilibiliComment(fileSystemDirectoryHandle, url, fakeThreadPool) {

    }

    async function xiaohongshuArticleAndComment(fileSystemDirectoryHandle, url, fakeThreadPool) {
      const urls = await getFromNewTab(url, async () => {
        await wait(4000)
        const urls = []
        for (let t = 0; t < 0; t++) {
          window.scrollBy(0, window.innerHeight)
          await wait(800)
        }
        return [...document.querySelectorAll('a.cover.mask.ld')].map(a => a.href)
      })
      console.log(urls)
      const writableFileStream = await fileSystemDirectoryHandle
        .getFileHandle('xiaohongshu.json', { create: true })
        .then(fileHandle => fileHandle.createWritable())
      await writableFileStream.write('[')
      let isFirstData = true
      await Promise.allSettled(urls.map(async subUrl => {
        const data = await getFromNewTab(subUrl, async () => {
          const data = {}
          await gather(() => document.querySelector("#noteContainer"))
          data.title = document?.querySelector('#detail-title')?.innerText ?? ''
          data.desc = document?.querySelector('#detail-desc > span > span:nth-child(1)')?.innerText ?? ''
          data.tags = [...document.querySelectorAll("#hash-tag")].map(ele => ele.innerText)
          while (document.querySelector('div.end-container') === null && document.querySelector('p.no-comments-text') === null) {
            document.querySelector('#noteContainer > div.interaction-container > div.note-scroller').scrollBy(0, window.innerHeight)
            await wait(500)
          }
          data.content = [...document.querySelectorAll('div.content')].map(ele => ele.innerText)
          return data
        }, fakeThreadPool)
        console.log(data)
        if (isFirstData) {
          await writableFileStream.write(JSON.stringify(data))
          isFirstData = false
        } else
          await writableFileStream.write(',' + JSON.stringify(data))
      }))
      await writableFileStream.write(']')
      await writableFileStream.close()
    }

    const matchUrls = [
      { reg: /^https:\/\/www\.bilibili\.com\/video\/.*/, func: bilibiliVideo },
      { reg: /^https:\/\/space\.bilibili\.com\/.*\/lists.*/, func: bilibiliCollection },
      { reg: /^https:\/\/meeting\.tencent\.com\/user-center\/shared-record-info.*/, func: tencentMeetingRecord },
      { reg: /^https:\/\/mp\.weixin\.qq\.com\/s\/.*/, func: weChatOfficialAccount },
      { reg: /^1/, func: weChatOfficialAccountImages },
      { reg: /^2/, func: bilibiliComment },
      { reg: /^https:\/\/www\.xiaohongshu\.com\/search_result.*/, func: xiaohongshuArticleAndComment }
    ]

    const urlInput = document.querySelector('#url')
    urlInput.focus()
    const numInput = document.querySelector('#num')
    numInput.addEventListener('change', () => { if (!numInput.checkValidity()) numInput.value = '' })
    const button = document.querySelector('#run')
    button.addEventListener('click', async () => {
      button.disabled = true
      const url = urlInput.value
      const fakeThreadPool = new FakeThreadPool(parseInt(numInput.value) || 1)
      root.insertAdjacentHTML('beforeend', '<h4>开始下载</h3>')
      const matched = matchUrls.find(({ reg }) => reg.test(url))
      if (matched) {
        const { func } = matched
        const fileSystemDirectoryHandle = await window.showDirectoryPicker({ mode: 'readwrite', startIn: 'downloads' }).catch(() => { })
        if (fileSystemDirectoryHandle) {
          await matched.func(fileSystemDirectoryHandle, url, fakeThreadPool)
            .then(() => root.insertAdjacentHTML('beforeend', '<h4>下载完成</h3><hr>'))
            .catch(err => root.insertAdjacentHTML('beforeend', `<h4>发生错误：</h3>${err}<hr>`))
        } else {
          root.insertAdjacentHTML('beforeend', '<h4>中断</h3><hr>')
        }
      } else {
        root.insertAdjacentHTML('beforeend', '<h4>不兼容该地址</h3><hr>')
      }
      button.disabled = false
    })

  </script>

</body>

</html>